<?php
include 'simplehtmldom/simple_html_dom.php';

class MySpaceCrawler {
	
	# actual page of the friendlist to be crawled
	var $page = 1; 
	
	# actual $html result string ( simple_html_dom object)
	var $html_result;
	
	# filestream object for output
	var $fileStream;
	
	var $actualDate;
	
	function __construct($filepath,$dbName,$dbServer='localhost',$dbPW='root', $dbUser='') {
     $this->fileStream = fopen($filepath, w);
	   $this->html_result = str_get_html('');
	   
	   $link = mysql_connect($dbServer, $dbPW, $dbUser);
	     
	   if (!$link) {
			die('connected to DB not successfully !!!' . mysql_error());
	   }
  
		$db_selected = mysql_select_db($dbName, $link);
		if (!$db_selected) {
			die ('No DB available : ' . mysql_error());
		}
		
		$this->actualDate = date("Y-m-d H:i:s");
	   
	}
	
	function __destruct() {
		$this->html_result->__destruct();
		unset($html_result);
		fclose($this->fileStream); 
	}

	
	#  parameter: friendID
	#  return: an array of friendIds from the actual page
	function getFriendIDs($friendID){
		# unset last html_result object to free memory:
		$this->html_result->__destruct();
		unset($this->html_result); 
		
		# new query: 
		$this->html_result = str_get_html( $this->queryFriends($friendID) ) ; 
		
		# find FriendID on a page: 
		foreach($this->html_result->find("div[friendid]") as $e){
			$this->print_out($e->getAttribute('friendid') . "\n",true);
			$out[] = $e->getAttribute('friendid');
		}
		return $out;
	}
	
	
	#  parameter: friendID, start page 
	#  return: an array of friendIds from the start page to end page
	function getAllFriendIDs($friendID, $startPage){
		
		$this->page = $startPage;
		$n=1;
		# do for all pages:
		do{		
			echo ".";

			if($n % 50 == 0)echo ($n * 40) ."\n		";$n++;
			
			
			$this->print_out('page: ' . $this->page . "--------------------------------- \n",true);
			
			$friendsOnPage = $this->getFriendIDs($friendID);
			
			# For each FriendID on a page:
			foreach($friendsOnPage as $f){
				$totalFriendsArray[] = $f;
			} 
			
			# If this is the first page we crawl, then count the Number of Friends:
			if($this->page == $startPage + 1){
				$totalFriends  = $this->getNumberOfFriends();
			}
		}while($this->page  <= (($totalFriends / 40) + 1));
		
		# set page back to default
		$this->page = 1; 
		
		return $totalFriendsArray;
	}
	
	
	#  parameter:
	#  return: Number of Friends of a friend-list-page, extracted from the actual $html_result string
	function getNumberOfFriends(){
		$e = $this->html_result->find("span[class=pgerTitle]",0);
		$temp = explode(' of ', $e->plaintext);
		return intval(str_replace( ",", "", $temp[1] )); //parse as a number
	}
	
	
	#  parameter: friendID
	#  return: html source from a profile
	function queryProfile($friendID){
		return $this->cURL('http://profile.myspace.com/index.cfm?fuseaction=user.viewprofile&friendid=' .$friendID );
	}
	
	
	#  parameter: friendID
	#  return: html source from a friend-list
	function queryFriends($friendID){
		 return $this->cURL('http://friends.myspace.com/index.cfm?fuseaction=user.viewfriends&friendId=' .$friendID. '&p=' . $this->page++); //increase page automaticly
	}
	
	#  parameter: friendID
	#  return: html source from a friend-list
	function queryProfileSnipplet($friendID){
		 return $this->cURL('http://friends.myspace.com/Modules/Common/Services/FriendToolTipService.asmx/GetFriendHelperData?userid=' . $friendID); //increase page automaticly
	}


	
	#  parameter: URL string
	#  return: a query result string
	function cURL($url,$retry = 5){
		$c = curl_init($url);
		
		# curl options:
		curl_setopt($c, CURLOPT_COOKIE,'MSCulture=PreferredCulture=en-US;'); // We want english results
		curl_setopt($c, CURLOPT_RETURNTRANSFER,1); 
		curl_setopt($c, CURLOPT_HEADER, 0);
		curl_setopt($c, CURLOPT_FOLLOWLOCATION, 1); // allow redirect
		curl_setopt($c, CURLOPT_TIMEOUT, 15); 
		
		
		# list of possible user agents:
		$agents[] = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; WOW64; SLCC1; .NET CLR 2.0.50727; .NET CLR 3.0.04506; Media Center PC 5.0)";
		$agents[] = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)";
		$agents[] = "Opera/9.63 (Windows NT 6.0; U; ru) Presto/2.1.1?";
		$agents[] = "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.0.5) Gecko/2008120122 Firefox/3.0.5?";
		$agents[] = "Mozilla/5.0 (X11; U; Linux i686 (x86_64); en-US; rv:1.8.1.18) Gecko/20081203 Firefox/2.0.0.18?";
		$agents[] = "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.16) Gecko/20080702 Firefox/2.0.0.16?";
		$agents[] = "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-us) AppleWebKit/525.27.1 (KHTML, like Gecko) Version/3.2.1 Safari/525.27.1?";
		$agents[] = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 1.1.4322; .NET CLR 3.0.04506.30; .NET CLR 3.0.04506.648)"; 
		$agents[] = "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5"; 
		$agents[] = "Opera/9.80 (Windows NT 5.1; U; pl) Presto/2.2.15 Version/10.10"; 
		
		# pick out a random user agent: 
		curl_setopt($c, CURLOPT_USERAGENT, array_rand($agents));
		
		# using a Proxy ?
		//curl_setopt($c, CURLOPT_PROXY, '216.217.98.100:8000');
		
		$result = curl_exec($c);
		if($result === false)
		{
			$this->print_out("\n Curl-Fehler: " . curl_error($c) . "\n", true);
			if($retry > 0) $this->cURL($url,$retry--);
		}
		
		
		curl_close($c);
		
		unset($c);
		
		return $result;
	}
	
	#  parameter: friendID
	#  save in file and echo
	function print_out($string,$beQuiet = false){
		if(!$beQuiet)echo $string;
		fwrite ($this->fileStream, $string);
	}
	
	
	#  parameter: friendID
	#  return: an array with basic attributes extracted from profile page
	function getBasicProfileInfo($friendID){
		# unset last html_result object to free memory:
		$this->html_result->__destruct();
		unset($this->html_result);
		
		# new query:
		$this->html_result = str_get_html( $this->queryProfile($friendID) ) ;
		
		# set base attributes:	
		$basicProfileInfo['friendID'] = $friendID;
		$basicProfileInfo['url'] = 'http://profile.myspace.com/index.cfm?fuseaction=user.viewprofile&friendid=' . $friendID;
		
		#	different types of html source code 
		#	have to be parsed differently:
		
		#	SourceType 1: 
		#	The easy one. Everything is accessable through classes
																	 
		if($r = $this->html_result->find('.profileUserInfo',0))
		{
			$basicProfileInfo['sourceType'] = 1;
			$basicProfileInfo['name'] = trim($r->find(".profileFullName fn")->plaintext );
			$basicProfileInfo['age'] = $this->parseAge( $r->find(".age",0)->plaintext );
			$basicProfileInfo['gender'] = $this->parseGender( $r->find(".gender",0)->plaintext );
			$basicProfileInfo['city'] = $this->parseCity( $r->find(".locality",0)->plaintext );
			$basicProfileInfo['region'] = $this->parseRegion( $r->find(".region",0)->plaintext );
			$basicProfileInfo['country'] = $this->parseCountry( $r->find(".country-name",0)->plaintext );
			$basicProfileInfo['lastlogin'] = $this->parseLastLogin( $r->find(".lastlogin",0)->plaintext );	
		} 
		
		#	SourceType 2: 
		#	The unstructered. Attributes are loosely seperated by <br/> tags 
																				
		elseif($r = $this->html_result->find('.profileInfo', 0))
		{
			$basicProfileInfo['sourceType'] = 2;
			$basicProfileInfo['name'] = trim($r->find(".nametext",0)->plaintext);
			
			# find table2: 						
			$r = $r->getElementById("Table2")->find(".text",2);
			
			# split text by <br\> and <br \>:						
			$r = preg_split('/<br\s?\/>/im', $r->innertext);
			
			# find the gender attribut position:									
			$genderPosition = 0;
			foreach($r as $i){
				if($this->parseGender($i) != false)break; // yes! we have it
				$genderPosition++;
			}
			
			# find other Attributes relative to the position of the gender Attribut: 																			
			$basicProfileInfo['age'] = $this->parseAge($r[$genderPosition + 1]);
			$basicProfileInfo['gender'] = $this->parseGender($r[$genderPosition]);
			$basicProfileInfo['city'] = $this->parseCity( strtok($r[$genderPosition + 2] , ',') );
			$basicProfileInfo['region'] = $this->parseRegion( strtok(',') );
			$basicProfileInfo['country'] = $this->parseCountry( $r[$genderPosition + 3] );
			$basicProfileInfo['lastlogin'] = $this->parseLastLogin($r[$genderPosition + 7]);
			
			# sometimes the city is not set; in this case it should be the region
			if(empty($basicProfileInfo['region'])){
				$basicProfileInfo['region'] = $basicProfileInfo['city'];
				$basicProfileInfo['city'] = '';
			}
		}
		
		#	SourceType 3: 
		#	The artist: What's going on here ? 
			
		elseif($r = $this->html_result->find('font[face=Arial, Helvetic, sans-serif]',0))
		{
			$basicProfileInfo['sourceType'] = 3;
			/*TODO ....*/
		}
		
		#	SourceType 3: 
		#	The unknown  
		
		else
		{
			$basicProfileInfo['sourceType'] = 'unknown !!!!!!!!!!!!!!!';
		}

		return $basicProfileInfo;
	}
	
	#  parameter: friendID
	#  return: an array with basic attributes extracted from an ajax snipplet page -> very fast, but with less information !!!
	function getShortBasicProfileInfo($friendID){
		# unset last html_result object to free memory:
		$this->html_result->__destruct();
		unset($this->html_result);
		
		# new query:
		$this->html_result = str_get_html( $this->queryProfileSnipplet($friendID) ) ;
		
		$this->html_result = str_get_html(htmlspecialchars_decode($this->html_result->find('displaytext',0)->innertext));
		//echo $this->html_result->outertext;
		$basicProfileInfo['friendID'] = $friendID;
		$basicProfileInfo['url'] = 'http://profile.myspace.com/index.cfm?fuseaction=user.viewprofile&friendid=' . $friendID;
		$basicProfileInfo['name'] = $this->html_result->find('.name',0)->plaintext;
		$basicProfileInfo['realName'] = $this->html_result->find('.realName',0)->plaintext;
		$this->html_result->find('.name',0)->outertext = '';
		$this->html_result->find('.realName',0)->outertext  = '';
		
		# split text by <br\> and <br \>:						
		$r = preg_split('/<br\s?\/>/im', $this->html_result->innertext);
		
		//echo $this->html_result->innertext;
		if($r[0] == 'Band'){
			
			$basicProfileInfo['sourceType'] = 'band';
			$basicProfileInfo['city'] = $this->parseCity( strtok($r[1] , ',') );
			$basicProfileInfo['region'] = $this->parseRegion( strtok(',') );
		
		}else{
			
			# find the gender attribut position:									
			$genderPosition = 0;
			foreach($r as $i){
				if($this->parseGender($i) != false)break; // yes! we have it
				$genderPosition++;
			}
			
			$basicProfileInfo['age'] = $this->parseAge( $r[$genderPosition - 1] );
			$basicProfileInfo['gender'] = $this->parseGender($r[$genderPosition]);
			$basicProfileInfo['city'] = $this->parseCity( strtok($r[$genderPosition + 1] , ',') );
			$basicProfileInfo['region'] = $this->parseRegion( strtok(',') );			
		
		}

			# sometimes the city is not set; in this case it should be the region
			if(empty($basicProfileInfo['region'])){
				$basicProfileInfo['region'] = $basicProfileInfo['city'];
				$basicProfileInfo['city'] = '';
			}
		
		return $basicProfileInfo;
	}
	

	function parseLastLogin($string){
		if(preg_match ('/\d{1,2}\/\d{1,2}\/\d{2,4}/im', $string, $match)){
			return $match[0];
		}else{
			return false;
		}
	}
	
	function parseAge($string){
		if(preg_match ('/[1-9]?[0-9]/im', $string, $match)){
			return $match[0];
		}else{
			return false;
		}
	}
	
	function parseGender($string){
		if(preg_match('/(male|female)/im', $string, $match)){
			return strtolower($match[0]);
		}else{
			return false;
		}
	}
	
	function parseCity($string){
			return trim(strtolower($string));
	}
	
	function parseRegion($string){
			return trim(strtolower($string));
	}
	
	function parseCountry($string){
			return trim(strtolower($string));
	}
	
	function clearHTML(){
		# delete all script tags
		foreach($this->html_result->find('script') as $h){
			$h->outertext = '';
		}
		# delete all style tags
		foreach($this->html_result->find('style') as $h){
			$h->outertext = '';
		}
	}
	
	function writeHTMLtoFile($basePath){
		$file = fopen($basePath . $friendID . '.html', w);
		$this->clearHTML();
		fwrite($file, $crawler->html_result);
		fclose ($file); 
		unset($file);
	}
	
	#parameter: actual FriendID (the one who has friends), array of FriendIDs from his/her friends
	#return: void
	function insertFriendLinksInCondor($friendID,$friendIDs, $dataset=0){
		foreach($friendIDs as $friend){
			$sql="INSERT INTO links (datasetId, link_from, link_to, link_time) 
					  VALUES ('". $dataset . "',  
							  '" . $friendID . "',
							  '". $friend ."',
							  '". $this->actualDate ."')";
			$result = mysql_query($sql);
		}
		
		return mysql_error() ? mysql_error()  : "OK";
	}
	
	
	#parameter: basicProfileInfo array (typically from getBasicProfileInfo)
	#return: void
	function insertProfileInCondor($basicProfileInfo, $dataset=0){	
		$sql="INSERT INTO chars (datasetId, url, site, city, region, age, username, gender) 
		  VALUES ('". $dataset . "',  
				  '" .$basicProfileInfo['friendID'].  "',  
				  '" .$basicProfileInfo['friendID'].  "',
				  '" .$basicProfileInfo['city']. "',
				  '". $basicProfileInfo['region']. "',
				  '". $basicProfileInfo['age'] ."',
				  '". $basicProfileInfo['name']. "',
				  '". $basicProfileInfo['gender']. "')";
		$result = mysql_query($sql);
		
		return mysql_error() ? mysql_error() : "OK";  
	}
	
	
	#parameter: start and end position of Artists you want to crawl from the US country charts (unsigned)
	#retun: array of friendIDs
	function getArtistsFromUnsignedCharts($startPos, $endPos){
		
		# generate parameter rankid, depeneding on how many Artists to crawl
		$rankid = 1;
		if($endPos <= 10)$rankid = 1;
		elseif($endPos > 10  && $endPos <= 50)	$rankid = 2;
		elseif($endPos > 50  && $endPos <= 100)	$rankid = 3;
		elseif($endPos > 100 && $endPos <= 150)	$rankid = 4;
		elseif($endPos > 150)$rankid = 5;

		// # new query:
 		$c = $this->cURL('http://music.myspace.com/index.cfm?fuseaction=music.topcharts&territoryid=US&genreid=11&typeid=0&labelid=1&rankid=' . $rankid); //increase page automaticly
		
		$this->html_result = str_get_html( $c ) ;
		foreach($this->html_result->find('.addToFriend') as $e) {

			if(preg_match ('/friendId=[0-9]+/im', $e->outertext, $match)){
				$i++;
				if($i >= $startPos)$out[] =  trim(str_replace('friendId=','',$match[0]));
			}
			
			if($i >= $endPos)break;
		}
		return $out;
	}
	
	function getArtistsFromCondorDataset(){
		
		$result = mysql_query('Select dataset,datasetid from datasets order by datasetid desc');

		while($row = mysql_fetch_array($result)){
			$i++;
			$temp = preg_match('/\([0-9]+\)/im',$row['dataset'],$match);
			$artists[$i]['friendID'] = preg_replace('/(\(|\))/im','',$match[0]); 
			$artists[$i]['datasetID'] = $row['datasetid'];
		}
		return	$artists;
	}
	
	function getFriendListFromCondor($friendID){
		$result = mysql_query("Select link_to from links where link_from = '$friendID' ");

		while($row = mysql_fetch_array($result)){
			$friends[] = $row['link_to'];
		}
		return	$friends;
	}
	
	function writeUnsignedArtistsCharts($friendID,$pos){
		mysql_query("Insert into charts (date, pos, artist) VALUES('". $this->actualDate. "','". $pos . "','" .$friendID . "')"); 
	}
	
}



?>